{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:00:33.234794Z",
     "start_time": "2025-01-23T08:00:30.839118Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-25T01:34:20.146973Z",
     "iopub.status.busy": "2025-01-25T01:34:20.146555Z",
     "iopub.status.idle": "2025-01-25T01:34:25.079977Z",
     "shell.execute_reply": "2025-01-25T01:34:25.079480Z",
     "shell.execute_reply.started": "2025-01-25T01:34:20.146950Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sys.version_info(major=3, minor=10, micro=14, releaselevel='final', serial=0)\n",
      "matplotlib 3.10.0\n",
      "numpy 1.26.4\n",
      "pandas 2.2.3\n",
      "sklearn 1.6.0\n",
      "torch 2.5.1+cu124\n",
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, torch:\n",
    "    print(module.__name__, module.__version__)\n",
    "    \n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)\n",
    "\n",
    "seed = 42\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed_all(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-02T03:33:53.350580400Z",
     "start_time": "2024-05-02T03:33:52.447226500Z"
    },
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-01-25T01:34:30.914690Z",
     "iopub.status.busy": "2025-01-25T01:34:30.914243Z",
     "iopub.status.idle": "2025-01-25T01:34:32.899992Z",
     "shell.execute_reply": "2025-01-25T01:34:32.899457Z",
     "shell.execute_reply.started": "2025-01-25T01:34:30.914665Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2025-01-25 09:34:30--  https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt\n",
      "正在解析主机 storage.googleapis.com (storage.googleapis.com)... 142.251.33.123, 142.251.211.251, 142.250.69.219, ...\n",
      "正在连接 storage.googleapis.com (storage.googleapis.com)|142.251.33.123|:443... 已连接。\n",
      "已发出 HTTP 请求，正在等待回应... 200 OK\n",
      "长度： 1115394 (1.1M) [text/plain]\n",
      "正在保存至: ‘shakespeare.txt’\n",
      "\n",
      "shakespeare.txt     100%[===================>]   1.06M   987KB/s    用时 1.1s    \n",
      "\n",
      "2025-01-25 09:34:32 (987 KB/s) - 已保存 ‘shakespeare.txt’ [1115394/1115394])\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:00:56.509250Z",
     "start_time": "2025-01-23T08:00:56.504813Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-25T01:46:12.885049Z",
     "iopub.status.busy": "2025-01-25T01:46:12.884752Z",
     "iopub.status.idle": "2025-01-25T01:46:12.889147Z",
     "shell.execute_reply": "2025-01-25T01:46:12.888743Z",
     "shell.execute_reply.started": "2025-01-25T01:46:12.885026Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length 1115394\n",
      "First Citizen:\n",
      "Before we proceed any further, hear me speak.\n",
      "\n",
      "All:\n",
      "Speak, speak.\n",
      "\n",
      "First Citizen:\n",
      "You\n"
     ]
    }
   ],
   "source": [
    "# https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt\n",
    "#文件已经下载好了\n",
    "with open(\"./shakespeare.txt\", \"r\", encoding=\"utf8\") as file:\n",
    "    text = file.read()\n",
    "\n",
    "print(\"length\", len(text))\n",
    "print(text[0:100])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 构造字典"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:01:28.777548Z",
     "start_time": "2025-01-23T08:01:28.769444Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-25T01:46:19.602246Z",
     "iopub.status.busy": "2025-01-25T01:46:19.601951Z",
     "iopub.status.idle": "2025-01-25T01:46:19.620139Z",
     "shell.execute_reply": "2025-01-25T01:46:19.619725Z",
     "shell.execute_reply.started": "2025-01-25T01:46:19.602223Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65\n",
      "['\\n', ' ', '!', '$', '&', \"'\", ',', '-', '.', '3', ':', ';', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']\n"
     ]
    }
   ],
   "source": [
    "# 1. generate vocab\n",
    "# 2. build mapping char->id\n",
    "# 3. data -> id_data  把数据都转为id\n",
    "# 4. a b c d [EOS] -> [BOS] b c d  预测下一个字符生成的模型，也就是输入是a，输出就是b\n",
    "\n",
    "#去重，留下独立字符，并排序（排序是为了好看）\n",
    "vocab = sorted(set(text))\n",
    "print(len(vocab))\n",
    "print(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:01:36.496656Z",
     "start_time": "2025-01-23T08:01:36.493637Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-25T01:46:49.958030Z",
     "iopub.status.busy": "2025-01-25T01:46:49.957723Z",
     "iopub.status.idle": "2025-01-25T01:46:49.961323Z",
     "shell.execute_reply": "2025-01-25T01:46:49.960763Z",
     "shell.execute_reply.started": "2025-01-25T01:46:49.958010Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 h\n",
      "1 o\n",
      "2 w\n"
     ]
    }
   ],
   "source": [
    "for idx,char in enumerate(['h','o','w']):\n",
    "    print(idx,char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:01:43.290659Z",
     "start_time": "2025-01-23T08:01:43.288068Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-25T01:47:16.756765Z",
     "iopub.status.busy": "2025-01-25T01:47:16.756416Z",
     "iopub.status.idle": "2025-01-25T01:47:16.760180Z",
     "shell.execute_reply": "2025-01-25T01:47:16.759570Z",
     "shell.execute_reply.started": "2025-01-25T01:47:16.756744Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'\\n': 0, ' ': 1, '!': 2, '$': 3, '&': 4, \"'\": 5, ',': 6, '-': 7, '.': 8, '3': 9, ':': 10, ';': 11, '?': 12, 'A': 13, 'B': 14, 'C': 15, 'D': 16, 'E': 17, 'F': 18, 'G': 19, 'H': 20, 'I': 21, 'J': 22, 'K': 23, 'L': 24, 'M': 25, 'N': 26, 'O': 27, 'P': 28, 'Q': 29, 'R': 30, 'S': 31, 'T': 32, 'U': 33, 'V': 34, 'W': 35, 'X': 36, 'Y': 37, 'Z': 38, 'a': 39, 'b': 40, 'c': 41, 'd': 42, 'e': 43, 'f': 44, 'g': 45, 'h': 46, 'i': 47, 'j': 48, 'k': 49, 'l': 50, 'm': 51, 'n': 52, 'o': 53, 'p': 54, 'q': 55, 'r': 56, 's': 57, 't': 58, 'u': 59, 'v': 60, 'w': 61, 'x': 62, 'y': 63, 'z': 64}\n"
     ]
    }
   ],
   "source": [
    "#每个字符都编好号，enumerate对每一个位置编号，生成的是列表中是元组，下面字典生成式\n",
    "char2idx = {char:idx for idx, char in enumerate(vocab)}\n",
    "print(char2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:02:04.850128Z",
     "start_time": "2025-01-23T08:02:04.847295Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-25T02:08:06.515983Z",
     "iopub.status.busy": "2025-01-25T02:08:06.515667Z",
     "iopub.status.idle": "2025-01-25T02:08:06.519089Z",
     "shell.execute_reply": "2025-01-25T02:08:06.518591Z",
     "shell.execute_reply.started": "2025-01-25T02:08:06.515964Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\\n' ' ' '!' '$' '&' \"'\" ',' '-' '.' '3' ':' ';' '?' 'A' 'B' 'C' 'D' 'E'\n",
      " 'F' 'G' 'H' 'I' 'J' 'K' 'L' 'M' 'N' 'O' 'P' 'Q' 'R' 'S' 'T' 'U' 'V' 'W'\n",
      " 'X' 'Y' 'Z' 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o'\n",
      " 'p' 'q' 'r' 's' 't' 'u' 'v' 'w' 'x' 'y' 'z']\n"
     ]
    }
   ],
   "source": [
    "# 把vocab从列表变为ndarray\n",
    "idx2char = np.array(vocab)\n",
    "print(idx2char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:03:03.944020Z",
     "start_time": "2025-01-23T08:03:03.874617Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-25T01:49:51.780521Z",
     "iopub.status.busy": "2025-01-25T01:49:51.780195Z",
     "iopub.status.idle": "2025-01-25T01:49:51.862798Z",
     "shell.execute_reply": "2025-01-25T01:49:51.862334Z",
     "shell.execute_reply.started": "2025-01-25T01:49:51.780501Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1115394,)\n",
      "1115394\n",
      "[18 47 56 57 58  1 15 47 58 47]\n",
      "First Citi\n"
     ]
    }
   ],
   "source": [
    "#把字符都转换为id\n",
    "text_as_int = np.array([char2idx[c] for c in text])\n",
    "print(text_as_int.shape)\n",
    "print(len(text_as_int))\n",
    "print(text_as_int[0:10])\n",
    "print(text[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:03:33.490881Z",
     "start_time": "2025-01-23T08:03:33.487237Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-25T01:50:46.504040Z",
     "iopub.status.busy": "2025-01-25T01:50:46.503717Z",
     "iopub.status.idle": "2025-01-25T01:50:46.507334Z",
     "shell.execute_reply": "2025-01-25T01:50:46.506911Z",
     "shell.execute_reply.started": "2025-01-25T01:50:46.504018Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11043"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1115394//101"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 把莎士比亚文集分成一个一个的样本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:09:37.251399Z",
     "start_time": "2025-01-23T08:09:37.247840Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-25T01:58:58.384824Z",
     "iopub.status.busy": "2025-01-25T01:58:58.384392Z",
     "iopub.status.idle": "2025-01-25T01:58:58.390210Z",
     "shell.execute_reply": "2025-01-25T01:58:58.389741Z",
     "shell.execute_reply.started": "2025-01-25T01:58:58.384802Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "class CharDataset(Dataset):\n",
    "    #text_as_int是字符的id列表，seq_length是每个样本的长度\n",
    "    def __init__(self, text_as_int, seq_length):\n",
    "        self.sub_len = seq_length + 1 #一个样本的长度\n",
    "        self.text_as_int = text_as_int\n",
    "        self.num_seq = len(text_as_int) // self.sub_len #样本的个数\n",
    "        \n",
    "    def __getitem__(self, index):#index是样本的索引，返回的是一个样本，比如第一个，就是0-100的字符,总计101个字符\n",
    "        return self.text_as_int[index * self.sub_len: (index + 1) * self.sub_len]\n",
    "    \n",
    "    def __len__(self): #返回样本的个数\n",
    "        return self.num_seq\n",
    "\n",
    "#batch是一个列表，列表中的每一个元素是一个样本，有101个字符，前100个是输入，后100个是输出\n",
    "def collat_fct(batch):\n",
    "    src_list = [] #输入\n",
    "    trg_list = [] #输出\n",
    "    for part in batch:\n",
    "        src_list.append(part[:-1]) #输入\n",
    "        trg_list.append(part[1:]) #输出\n",
    "        \n",
    "    src_list = np.array(src_list) #把列表转换为ndarray\n",
    "    trg_list = np.array(trg_list) #把列表转换为ndarray\n",
    "    return torch.Tensor(src_list).to(dtype=torch.int64), torch.Tensor(trg_list).to(dtype=torch.int64) #返回的是一个元组，元组中的每一个元素是一个torch.Tensor\n",
    "        \n",
    "#每个样本的长度是101，也就是100个字符+1个结束符\n",
    "train_ds = CharDataset(text_as_int, 100)\n",
    "train_dl = DataLoader(train_ds, batch_size=64, shuffle=True, collate_fn=collat_fct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:09:39.865440Z",
     "start_time": "2025-01-23T08:09:39.859267Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-25T01:59:01.446962Z",
     "iopub.status.busy": "2025-01-25T01:59:01.446662Z",
     "iopub.status.idle": "2025-01-25T01:59:01.487826Z",
     "shell.execute_reply": "2025-01-25T01:59:01.487406Z",
     "shell.execute_reply.started": "2025-01-25T01:59:01.446940Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([64, 100])\n",
      "torch.Size([64, 100])\n"
     ]
    }
   ],
   "source": [
    "for datas, labels in train_dl:\n",
    "    print(datas.shape)\n",
    "    print(labels.shape)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:13:43.092169Z",
     "start_time": "2025-01-23T08:13:43.083620Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-25T01:59:11.101682Z",
     "iopub.status.busy": "2025-01-25T01:59:11.101364Z",
     "iopub.status.idle": "2025-01-25T01:59:11.117296Z",
     "shell.execute_reply": "2025-01-25T01:59:11.116887Z",
     "shell.execute_reply.started": "2025-01-25T01:59:11.101658Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=================================== 一层单向 RNN ===================================\n",
      "            embedding.weight            paramerters num: 16640\n",
      "            rnn.weight_ih_l0            paramerters num: 262144\n",
      "            rnn.weight_hh_l0            paramerters num: 1048576\n",
      "             rnn.bias_ih_l0             paramerters num: 1024\n",
      "             rnn.bias_hh_l0             paramerters num: 1024\n",
      "               fc.weight                paramerters num: 66560\n",
      "                fc.bias                 paramerters num: 65\n"
     ]
    }
   ],
   "source": [
    "class CharRNN(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim=256, hidden_dim=1024):\n",
    "        super(CharRNN, self).__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        #batch_first=True,输入的数据格式是(batch_size, seq_len, embedding_dim)\n",
    "        self.rnn = nn.RNN(embedding_dim, hidden_dim, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_dim, vocab_size)\n",
    "        \n",
    "    def forward(self, x, hidden=None):\n",
    "        x = self.embedding(x) #(batch_size, seq_len) -> (batch_size, seq_len, embedding_dim) (64, 100, 256)\n",
    "        #这里和02的差异是没有只拿最后一个输出，而是把所有的输出都拿出来了\n",
    "        #(batch_size, seq_len, embedding_dim)->(batch_size, seq_len, hidden_dim)(64, 100, 1024)\n",
    "        output, hidden = self.rnn(x, hidden)\n",
    "        x = self.fc(output) #[bs, seq_len, hidden_dim]--->[bs, seq_len, vocab_size] (64, 100,65)\n",
    "        return x, hidden #x的shape是(batch_size, seq_len, vocab_size)\n",
    "    \n",
    "    \n",
    "vocab_size = len(vocab)\n",
    "\n",
    "print(\"{:=^80}\".format(\" 一层单向 RNN \"))       \n",
    "for key, value in CharRNN(vocab_size).named_parameters():\n",
    "    print(f\"{key:^40}paramerters num: {np.prod(value.shape)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:15:23.555522Z",
     "start_time": "2025-01-23T08:15:23.551754Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-25T02:02:35.257081Z",
     "iopub.status.busy": "2025-01-25T02:02:35.256781Z",
     "iopub.status.idle": "2025-01-25T02:02:35.260979Z",
     "shell.execute_reply": "2025-01-25T02:02:35.260590Z",
     "shell.execute_reply.started": "2025-01-25T02:02:35.257060Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16640\n",
      "262144\n",
      "1048576\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "66560"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(65*256)\n",
    "print(256*1024)\n",
    "print(1024*1024)\n",
    "1024*65"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:27:35.393573400Z",
     "start_time": "2024-07-29T07:27:35.327680900Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-25T02:04:52.807678Z",
     "iopub.status.busy": "2025-01-25T02:04:52.807125Z",
     "iopub.status.idle": "2025-01-25T02:04:52.835736Z",
     "shell.execute_reply": "2025-01-25T02:04:52.835291Z",
     "shell.execute_reply.started": "2025-01-25T02:04:52.807656Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 100])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 100, 65])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_inputs = torch.randint(0, vocab_size, (3, 100))\n",
    "print(sample_inputs.shape)\n",
    "model = CharRNN(vocab_size)\n",
    "output=model(sample_inputs)\n",
    "output[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:57:28.992008300Z",
     "start_time": "2024-07-29T07:57:28.981317Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-25T02:04:54.033438Z",
     "iopub.status.busy": "2025-01-25T02:04:54.033115Z",
     "iopub.status.idle": "2025-01-25T02:04:54.036931Z",
     "shell.execute_reply": "2025-01-25T02:04:54.036519Z",
     "shell.execute_reply.started": "2025-01-25T02:04:54.033417Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([300, 65])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output[0].reshape(-1, vocab_size).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:26:37.116153100Z",
     "start_time": "2024-07-29T07:26:37.108972300Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-25T02:04:54.918127Z",
     "iopub.status.busy": "2025-01-25T02:04:54.917817Z",
     "iopub.status.idle": "2025-01-25T02:04:54.921343Z",
     "shell.execute_reply": "2025-01-25T02:04:54.920950Z",
     "shell.execute_reply.started": "2025-01-25T02:04:54.918108Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "66560"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "1024*65"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:28:19.937860300Z",
     "start_time": "2024-07-29T07:28:19.932860Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-25T02:04:55.884946Z",
     "iopub.status.busy": "2025-01-25T02:04:55.884626Z",
     "iopub.status.idle": "2025-01-25T02:04:55.889512Z",
     "shell.execute_reply": "2025-01-25T02:04:55.889054Z",
     "shell.execute_reply.started": "2025-01-25T02:04:55.884925Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class SaveCheckpointsCallback:\n",
    "    def __init__(self, save_dir, save_step=5000, save_best_only=True):\n",
    "        \"\"\"\n",
    "        Save checkpoints each save_epoch epoch. \n",
    "        We save checkpoint by epoch in this implementation.\n",
    "        Usually, training scripts with pytorch evaluating model and save checkpoint by step.\n",
    "\n",
    "        Args:\n",
    "            save_dir (str): dir to save checkpoint\n",
    "            save_epoch (int, optional): the frequency to save checkpoint. Defaults to 1.\n",
    "            save_best_only (bool, optional): If True, only save the best model or save each model at every epoch.\n",
    "        \"\"\"\n",
    "        self.save_dir = save_dir\n",
    "        self.save_step = save_step\n",
    "        self.save_best_only = save_best_only\n",
    "        self.best_metrics = -1\n",
    "        \n",
    "        # mkdir\n",
    "        if not os.path.exists(self.save_dir):\n",
    "            os.mkdir(self.save_dir)\n",
    "        \n",
    "    def __call__(self, step, state_dict, metric=None):\n",
    "        if step % self.save_step > 0:\n",
    "            return\n",
    "        \n",
    "        if self.save_best_only:\n",
    "            assert metric is not None\n",
    "            if metric >= self.best_metrics:\n",
    "                # save checkpoints\n",
    "                torch.save(state_dict, os.path.join(self.save_dir, \"best.ckpt\"))\n",
    "                # update best metrics\n",
    "                self.best_metrics = metric\n",
    "        else:\n",
    "            torch.save(state_dict, os.path.join(self.save_dir, f\"{step}.ckpt\"))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T08:16:19.788186Z",
     "start_time": "2025-01-23T08:16:18.860450Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-25T02:04:57.705682Z",
     "iopub.status.busy": "2025-01-25T02:04:57.705362Z",
     "iopub.status.idle": "2025-01-25T02:04:59.435430Z",
     "shell.execute_reply": "2025-01-25T02:04:59.434948Z",
     "shell.execute_reply.started": "2025-01-25T02:04:57.705661Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 训练\n",
    "def training(\n",
    "    model, \n",
    "    train_loader, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    save_ckpt_callback=None,\n",
    "    stateful=False      # 想用stateful，batch里的数据就必须连续，不能打乱\n",
    "    ):\n",
    "    record_dict = {\n",
    "        \"train\": [],\n",
    "    }\n",
    "    \n",
    "    global_step = 0\n",
    "    model.train()\n",
    "    hidden = None\n",
    "    with tqdm(total=epoch * len(train_loader)) as pbar:\n",
    "        for epoch_id in range(epoch):\n",
    "            # training\n",
    "            for datas, labels in train_loader:\n",
    "                datas = datas.to(device)\n",
    "                labels = labels.to(device)\n",
    "                # 梯度清空\n",
    "                optimizer.zero_grad()\n",
    "                # 模型前向计算,如果数据集打乱了，stateful=False，hidden就要清空\n",
    "                # 如果数据集没有打乱，stateful=True，hidden就不需要清空\n",
    "                logits, hidden = model(datas, hidden=hidden if stateful else None)\n",
    "                # 计算损失,交叉熵损失第一个参数要是二阶张量，第二个参数要是一阶张量，所以要reshape\n",
    "                loss = loss_fct(logits.reshape(-1, vocab_size), labels.reshape(-1))\n",
    "                # 梯度回传\n",
    "                loss.backward()\n",
    "                # 调整优化器，包括学习率的变动等\n",
    "                optimizer.step()\n",
    " \n",
    "                loss = loss.cpu().item()\n",
    "                # record\n",
    "                \n",
    "                record_dict[\"train\"].append({\n",
    "                    \"loss\": loss, \"step\": global_step\n",
    "                })\n",
    "   \n",
    "                # 保存模型权重 save model checkpoint\n",
    "                if save_ckpt_callback is not None:\n",
    "                    save_ckpt_callback(global_step, model.state_dict(), metric=-loss)\n",
    "                # udate step\n",
    "                global_step += 1\n",
    "                pbar.update(1)\n",
    "                pbar.set_postfix({\"epoch\": epoch_id})\n",
    "        \n",
    "    return record_dict\n",
    "        \n",
    "\n",
    "epoch = 100\n",
    "\n",
    "model = CharRNN(vocab_size=vocab_size)\n",
    "\n",
    "# 1. 定义损失函数 采用交叉熵损失 \n",
    "loss_fct = nn.CrossEntropyLoss()\n",
    "# 2. 定义优化器 采用 adam\n",
    "# Optimizers specified in the torch.optim package\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "\n",
    "# save best\n",
    "if not os.path.exists(\"checkpoints\"):\n",
    "    os.makedirs(\"checkpoints\")\n",
    "save_ckpt_callback = SaveCheckpointsCallback(\"checkpoints/text_generation\", save_step=1000, save_best_only=True)\n",
    "\n",
    "\n",
    "model = model.to(device)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-25T02:05:00.771132Z",
     "iopub.status.busy": "2025-01-25T02:05:00.770697Z",
     "iopub.status.idle": "2025-01-25T02:07:01.686093Z",
     "shell.execute_reply": "2025-01-25T02:07:01.685648Z",
     "shell.execute_reply.started": "2025-01-25T02:05:00.771110Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 17300/17300 [02:00<00:00, 143.08it/s, epoch=99]\n"
     ]
    }
   ],
   "source": [
    "record = training(\n",
    "    model,\n",
    "    train_dl,\n",
    "    epoch,\n",
    "    loss_fct,\n",
    "    optimizer,\n",
    "    save_ckpt_callback=save_ckpt_callback,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-25T02:07:06.276482Z",
     "iopub.status.busy": "2025-01-25T02:07:06.276163Z",
     "iopub.status.idle": "2025-01-25T02:07:06.401647Z",
     "shell.execute_reply": "2025-01-25T02:07:06.401233Z",
     "shell.execute_reply.started": "2025-01-25T02:07:06.276460Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAikAAAGdCAYAAADXIOPgAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAWrBJREFUeJzt3Xd4VFX+P/D3nclkkpBOeg8khBog1NCVDgqsLioWBLHDroqLLq6uoD+FFV10LahrwQYoLKBfRSACIZTQQkIntEAgpEBCMqmTycz5/TEzNxlTIAVzw7xfz8Mz5M69kzMfLpN3zj3nXEkIIUBERESkMKrWbgARERFRXRhSiIiISJEYUoiIiEiRGFKIiIhIkRhSiIiISJEYUoiIiEiRGFKIiIhIkRhSiIiISJEcWrsBN8JkMuHy5ctwc3ODJEmt3RwiIiK6AUIIFBcXIygoCCpV4/tF2kRIuXz5MkJDQ1u7GURERNQEFy9eREhISKOPaxMhxc3NDYD5Tbq7u7fY6xoMBmzevBljxoyBRqNpsddta1gH1gBgDaxYB9YAYA2smlsHnU6H0NBQ+ed4Y7WJkGK9xOPu7t7iIcXFxQXu7u52fxLaex1YA9bAinVgDQDWwKql6tDUoRocOEtERESKxJBCREREisSQQkRERIrEkEJERESKxJBCREREisSQQkRERIrEkEJERESKxJBCREREisSQQkRERIrEkEJERESKxJBCREREisSQQkRERIpk1yHly90X8L8MFdJzilu7KURERPQ7dh1SNhzNQVKOChevlbd2U4iIiOh37DqkWG8cbRKiVdtBREREtdl1SFFJ5pjCjEJERKQ8dh1SLBmFPSlEREQKZOchRbr+TkRERNQq7DqkqOSelNZtBxEREdVm1yGFA2eJiIiUy65DCgfOEhERKZddhxRrV4pgSiEiIlIcuw4pck9KK7eDiIiIarPrkMIxKURERMpl1yGFY1KIiIiUy65DCjgFmYiISLHsOqSo5LXcmFKIiIiUxq5DimTpSmFPChERkfLYdUhRyVOQW7cdREREVJtdhxTrvXs4u4eIiEh5mhVSFi9eDEmS8Oyzzza43+rVq9G5c2c4OTmhR48e2LBhQ3O+bYux3l+QEYWIiEh5mhxS9u/fj08++QSxsbEN7rd7925MmzYNs2bNQmpqKqZMmYIpU6bg6NGjTf3WLcY6bpYrzhIRESlPk0JKSUkJHnjgAfz3v/+Fl5dXg/u+9957GDduHObNm4cuXbrg9ddfR1xcHD744IMmNbglcZ0UIiIi5XJoykGzZ8/GxIkTMWrUKPy///f/Gtw3OTkZc+fOtdk2duxYrF+/vt5j9Ho99Hq9/LVOpwMAGAwGGAyGpjS5HuZ0YqgytvDrti3W984asAY1H+0V68AaAKyBVXPr0Nz6NTqkrFq1CgcPHsT+/ftvaP+cnBz4+/vbbPP390dOTk69xyxatAgLFy6stX3z5s1wcXFpXIMbkJujAqDCyfST2KA70WKv21YlJCS0dhNaHWvAGlixDqwBwBpYNbUOZWVlzfq+jQopFy9exDPPPIOEhAQ4OTk16xs3ZP78+Ta9LzqdDqGhoRgzZgzc3d1b7PtsLj4E5OeiU6cYTBjaocVet60xGAxISEjA6NGjodFoWrs5rYI1YA2sWAfWAGANrJpbB+uVkKZqVEhJSUlBXl4e4uLi5G1GoxFJSUn44IMPoNfroVarbY4JCAhAbm6uzbbc3FwEBATU+320Wi20Wm2t7RqNpkVPFrXKPCRHUqns+iS0aun6tkWsAWtgxTqwBgBrYNXUOjS3do0aODty5EgcOXIEaWlp8p++ffvigQceQFpaWq2AAgDx8fHYsmWLzbaEhATEx8c3q+EtgYu5ERERKVejelLc3NzQvXt3m23t2rVD+/bt5e3Tp09HcHAwFi1aBAB45plnMHz4cLzzzjuYOHEiVq1ahQMHDuDTTz9tobfQdJJ8g0GmFCIiIqVp8RVnMzMzkZ2dLX89aNAgrFixAp9++il69uyJNWvWYP369bXCTmuQOAWZiIhIsZo0BbmmxMTEBr8GgKlTp2Lq1KnN/VYtTl5xlimFiIhIcez63j3yYm6t3A4iIiKqza5DinVZfBNTChERkeLYd0iRx6QwpRARESmNnYcU8yMzChERkfLYdUiR10nhqBQiIiLFseuQIllGpXBMChERkfLYdUjhirNERETKZdchBRw4S0REpFh2HVKqx6QQERGR0th5SLGOSWFMISIiUhq7DinWxdyYUYiIiJTHvkMK74JMRESkWHYeUqTr70REREStwq5DikruSWnddhAREVFtdh1SqhdzY0ohIiJSGrsOKVzMjYiISLnsOqRADilMKUREREpj1yHFuk4KIwoREZHy2HVIsc7t4ZgUIiIi5bHrkCL3pDCjEBERKY5dhxRwCjIREZFi2XVIUcmLuTGlEBERKY1dh5TqMSmt2gwiIiKqg12HFK6TQkREpFx2HVKs9+7h7B4iIiLlsfOQYn5kRCEiIlIeuw4p8hRkDkohIiJSHLsOKexJISIiUi67DikqjkkhIiJSLLsOKVbMKERERMpj1yGFU5CJiIiUy65DiiTfBZkphYiISGnsOqSoeO8eIiIixbLrkFK9LD5TChERkdLYd0ixXu5hRiEiIlIcOw8p5kfBlEJERKQ4dh1S5BVnW7kdREREVJtdhxSOSSEiIlIu+w4pHJNCRESkWHYdUriYGxERkXI1KqQsW7YMsbGxcHd3h7u7O+Lj4/Hrr7/Wu//y5cshSZLNHycnp2Y3uqVU32CQKYWIiEhpHBqzc0hICBYvXozo6GgIIfDVV19h8uTJSE1NRbdu3eo8xt3dHenp6fLX1kssSlB9g8FWbggRERHV0qiQcuedd9p8/cYbb2DZsmXYs2dPvSFFkiQEBAQ0vYU3kTUu8XIPERGR8jQqpNRkNBqxevVqlJaWIj4+vt79SkpKEB4eDpPJhLi4OLz55pv1BhorvV4PvV4vf63T6QAABoMBBoOhqU2uxWQyAQCMJlOLvm5bY33vrAFrUPPRXrEOrAHAGlg1tw7NrZ8kGrmS2ZEjRxAfH4+Kigq4urpixYoVmDBhQp37Jicn4/Tp04iNjUVRURHefvttJCUl4dixYwgJCan3eyxYsAALFy6stX3FihVwcXFpTHMblHJVwten1Yh2N2FON1OLvS4REREBZWVluP/++1FUVAR3d/dGH9/okFJZWYnMzEwUFRVhzZo1+Oyzz7B9+3Z07dr1uscaDAZ06dIF06ZNw+uvv17vfnX1pISGhuLq1atNepP1+SntEp7/33H0D/fEd4/2b7HXbWsMBgMSEhIwevRoaDSa1m5Oq2ANWAMr1oE1AFgDq+bWQafTwcfHp8khpdGXexwdHREVFQUA6NOnD/bv34/33nsPn3zyyXWP1Wg06N27N86cOdPgflqtFlqtts7jW/JkcVBb3r4k2fVJaNXS9W2LWAPWwIp1YA0A1sCqqXVobu2avU6KyWSy6fVoiNFoxJEjRxAYGNjcb9sirBONuOIsERGR8jSqJ2X+/PkYP348wsLCUFxcjBUrViAxMRGbNm0CAEyfPh3BwcFYtGgRAOC1117DwIEDERUVhcLCQixZsgQXLlzAo48+2vLvpAmUNB2aiIiIbDUqpOTl5WH69OnIzs6Gh4cHYmNjsWnTJowePRoAkJmZCZWqunPm2rVreOyxx5CTkwMvLy/06dMHu3fvvqHxK38EldyT0rrtICIiotoaFVI+//zzBp9PTEy0+Xrp0qVYunRpoxv1R5FgXcyNKYWIiEhpeO8ecDE3IiIiJbLrkAI5pDClEBERKY1dhxTrvXsYUYiIiJTHrkMKpyATEREpl12HFLknhRmFiIhIcew6pEicgkxERKRY9h1SwOk9RERESmXXIYWLuRERESmXXYcU6+Uewfk9REREimPXIcU6cJY9KURERMpj1yHFikNSiIiIlMeuQ0r1FGSmFCIiIqWx65BSPSaFiIiIlMauQ0r1mBTGFCIiIqWx65Bi6UjhmBQiIiIFsu+QwrsgExERKZadhxTeBZmIiEip7DqkcMVZIiIi5bLrkGK9dw8HzhIRESmPfYcUeeRsqzaDiIiI6mDXIYVTkImIiJTLrkMKF3MjIiJSLrsOKdUDZxlTiIiIlMauQ4p14CwzChERkfLYd0iRF3Nr3XYQERFRbXYeUqyLuTGlEBERKY1dhxQu5kZERKRcdh1SeLmHiIhIuew8pFgHzjKlEBERKY19hxTLIyMKERGR8th1SOGKs0RERMpl1yGFY1KIiIiUy65DCntSiIiIlMuuQwoREREpl12HFK6TQkREpFx2HVIkXu4hIiJSLLsOKSoOnCUiIlIsuw4p1p4UgAu6ERERKY19h5Qaf2dGISIiUha7DimqGj0pHJdCRESkLI0KKcuWLUNsbCzc3d3h7u6O+Ph4/Prrrw0es3r1anTu3BlOTk7o0aMHNmzY0KwGt6QaGYVL4xMRESlMo0JKSEgIFi9ejJSUFBw4cAC33347Jk+ejGPHjtW5/+7duzFt2jTMmjULqampmDJlCqZMmYKjR4+2SOObS1UjpLAnhYiISFkaFVLuvPNOTJgwAdHR0ejUqRPeeOMNuLq6Ys+ePXXu/95772HcuHGYN28eunTpgtdffx1xcXH44IMPWqTxzVdz4GwrNoOIiIhqcWjqgUajEatXr0ZpaSni4+Pr3Cc5ORlz58612TZ27FisX7++wdfW6/XQ6/Xy1zqdDgBgMBhgMBia2uRajMbq16qsNEANU4u9dltirWlL1ratYQ1YAyvWgTUAWAOr5tahufVrdEg5cuQI4uPjUVFRAVdXV6xbtw5du3atc9+cnBz4+/vbbPP390dOTk6D32PRokVYuHBhre2bN2+Gi4tLY5tcL70RsJZg46ZN0Kpb7KXbpISEhNZuQqtjDVgDK9aBNQBYA6um1qGsrKxZ37fRISUmJgZpaWkoKirCmjVr8PDDD2P79u31BpWmmD9/vk0PjE6nQ2hoKMaMGQN3d/cW+z7FZRXAviQAwOgxY+CqbXLHUptmMBiQkJCA0aNHQ6PRtHZzWgVrwBpYsQ6sAcAaWDW3DtYrIU3V6J/Kjo6OiIqKAgD06dMH+/fvx3vvvYdPPvmk1r4BAQHIzc212Zabm4uAgIAGv4dWq4VWq621XaPRtOjJ4qgxyn93cHCw6xMRaPn6tkWsAWtgxTqwBgBrYNXUOjS3ds1eJ8VkMtmMH6kpPj4eW7ZssdmWkJBQ7xiWP5pks05KKzaEiIiIamlUT8r8+fMxfvx4hIWFobi4GCtWrEBiYiI2bdoEAJg+fTqCg4OxaNEiAMAzzzyD4cOH45133sHEiROxatUqHDhwAJ9++mnLv5MmkGyWnG21ZhAREVEdGhVS8vLyMH36dGRnZ8PDwwOxsbHYtGkTRo8eDQDIzMyESlXdOTNo0CCsWLECL7/8Ml566SVER0dj/fr16N69e8u+iybiirNERETK1aiQ8vnnnzf4fGJiYq1tU6dOxdSpUxvVqD8KO1KIiIiUy67v3SNxxVkiIiLFsvOQIkGy9KEwoxARESmLXYeUmgRTChERkaLYfUixXvLhFGQiIiJlYUixPAoOnSUiIlIUhhTLI3tSiIiIlIUhxfLIMSlERETKwpBiSSnMKERERMpi9yHFiiGFiIhIWew+pFTP7mFKISIiUhKGFMsjIwoREZGyMKRYHtmTQkREpCwMKZZHZhQiIiJlYUiRZ/cwpRARESmJ3YcUK0YUIiIiZbH7kMLZPURERMpk9yHFWgBmFCIiImWx+5BixZ4UIiIiZbH7kMJl8YmIiJSJIcXyyJBCRESkLAwplkde7iEiIlIWhhTr5Z7WbQYRERH9DkOK5ZE9KURERMpi9yHFihmFiIhIWew+pHBZfCIiImViSLE8MqIQEREpC0OK5dFkYkwhIiJSEoYUzu4hIiJSJIYUyyNn9xARESmL3YcUGTMKERGRoth9SLFe7uGQFCIiImVhSLE8CnalEBERKQpDiuWRPSlERETKwpDCxdyIiIgUiSHF8siMQkREpCwMKZZHjkkhIiJSFoYU6+weU+u2g4iIiGwxpFge2Y9CRESkLHYfUqy44iwREZGy2H1I4eweIiIiZWJIsTwyoxARESlLo0LKokWL0K9fP7i5ucHPzw9TpkxBenp6g8csX74ckiTZ/HFycmpWo1sSF3MjIiJSpkaFlO3bt2P27NnYs2cPEhISYDAYMGbMGJSWljZ4nLu7O7Kzs+U/Fy5caFajW5J8uYdDZ4mIiBTFoTE7b9y40ebr5cuXw8/PDykpKRg2bFi9x0mShICAgKa18CaTIABI7EkhIiJSmEaFlN8rKioCAHh7eze4X0lJCcLDw2EymRAXF4c333wT3bp1q3d/vV4PvV4vf63T6QAABoMBBoOhOU22UfO1qqqqWvS12xLr+7bX9w+wBgBrYMU6sAYAa2DV3Do0t36SaOK0FpPJhEmTJqGwsBA7d+6sd7/k5GScPn0asbGxKCoqwttvv42kpCQcO3YMISEhdR6zYMECLFy4sNb2FStWwMXFpSnNrdeHx1U4VaTCQ1FG9PVldwoREVFLKSsrw/3334+ioiK4u7s3+vgmh5SnnnoKv/76K3bu3Flv2KiLwWBAly5dMG3aNLz++ut17lNXT0poaCiuXr3apDfZUFv+9N4WpBep8Pbd3TG5V1CLvXZbYjAYkJCQgNGjR0Oj0bR2c1oFa8AaWLEOrAHAGlg1tw46nQ4+Pj5NDilNutwzZ84c/Pzzz0hKSmpUQAEAjUaD3r1748yZM/Xuo9VqodVq6zy2pU8W6+weSaW26xMRuDn1bWtYA9bAinVgDQDWwKqpdWhu7Ro1u0cIgTlz5mDdunXYunUrIiMjG/0NjUYjjhw5gsDAwEYfezNUz+4hIiIiJWlUT8rs2bOxYsUK/Pjjj3Bzc0NOTg4AwMPDA87OzgCA6dOnIzg4GIsWLQIAvPbaaxg4cCCioqJQWFiIJUuW4MKFC3j00Udb+K00TfU6KYwpREREStKokLJs2TIAwIgRI2y2f/nll5gxYwYAIDMzEypVdQfNtWvX8NhjjyEnJwdeXl7o06cPdu/eja5duzav5S1E4h0GiYiIFKlRIeVGxtgmJibafL106VIsXbq0UY36I7EnhYiISJl47x7LIyMKERGRsth9SLFiTwoREZGy2H1IkWf3MKMQEREpCkOK5bGJa9oRERHRTcKQwnVSiIiIFIkhxfJo4m2QiYiIFIUhxfLIjEJERKQsdh9SrJhRiIiIlMXuQ0r17B7GFCIiIiVhSLE8MqMQEREpC0OKJaVwMTciIiJlYUixPDKiEBERKQtDiuWRPSlERETKwpDCZfGJiIgUiSHF8sjZPURERMrCkGJ5ZEYhIiJSFrsPKZBn97RuM4iIiMiW3YeU6tk9TClERERKwpBieWRPChERkbIwpHBQChERkSIxpFge2ZNCRESkLAwplkeOSSEiIlIWuw8pnN1DRESkTHYfUrgsPhERkTIxpFj/woxCRESkKAwp8uUephQiIiIlYUixPDKjEBERKQtDiuWRA2eJiIiUhSHFklI4BZmIiEhZGFIsj7zcQ0REpCwMKZZHwZRCRESkKHYfUriYGxERkTLZfUiRLGNROCaFiIhIWRhSLI/sSSEiIlIWhhTr7B6GFCIiIkVhSLE8cuAsERGRsjCksCeFiIhIkew+pFgLYDCaWrUdREREZMvuQ4qzg/lRV2Fo3YYQERGRDbsPKe0sIeVaGUMKERGRkjQqpCxatAj9+vWDm5sb/Pz8MGXKFKSnp1/3uNWrV6Nz585wcnJCjx49sGHDhiY3uKW5WEJKYVll6zaEiIiIbDQqpGzfvh2zZ8/Gnj17kJCQAIPBgDFjxqC0tLTeY3bv3o1p06Zh1qxZSE1NxZQpUzBlyhQcPXq02Y1vCe0czCNmC9mTQkREpCgOjdl548aNNl8vX74cfn5+SElJwbBhw+o85r333sO4ceMwb948AMDrr7+OhIQEfPDBB/j444+b2OyWI/eklBsghIBkne5DRERErapRIeX3ioqKAADe3t717pOcnIy5c+fabBs7dizWr19f7zF6vR56vV7+WqfTAQAMBgMMhpbr8TAYDHJIMZoErpWUw81J02Kv31ZYa9qStW1rWAPWwIp1YA0A1sCquXVobv0k0cRVzEwmEyZNmoTCwkLs3Lmz3v0cHR3x1VdfYdq0afK2jz76CAsXLkRubm6dxyxYsAALFy6stX3FihVwcXFpSnMb9Le9ahhMEl7pXQUfpxZ/eSIiIrtUVlaG+++/H0VFRXB3d2/08U3uSZk9ezaOHj3aYEBpqvnz59v0vuh0OoSGhmLMmDFNepP1MRgMSEhIQHtXJ+To9Og9YDB6BHu02Ou3FdY6jB49GhqN/fUkAawBwBpYsQ6sAcAaWDW3DtYrIU3VpJAyZ84c/Pzzz0hKSkJISEiD+wYEBNTqMcnNzUVAQEC9x2i1Wmi12lrbNRrNTTlZPJ01yNHpUVwp7PpkvFn1bUtYA9bAinVgDQDWwKqpdWhu7Ro1u0cIgTlz5mDdunXYunUrIiMjr3tMfHw8tmzZYrMtISEB8fHxjWvpTeTpYi4ipyETEREpR6N6UmbPno0VK1bgxx9/hJubG3JycgAAHh4ecHZ2BgBMnz4dwcHBWLRoEQDgmWeewfDhw/HOO+9g4sSJWLVqFQ4cOIBPP/20hd9K03m6OALgNGQiIiIlaVRPyrJly1BUVIQRI0YgMDBQ/vP999/L+2RmZiI7O1v+etCgQVixYgU+/fRT9OzZE2vWrMH69evRvXv3lnsXzeThbO1JYUghIiJSikb1pNzIRKDExMRa26ZOnYqpU6c25lv9obwsl3uu8XIPERGRYtj9vXuA6p6UonL2pBARESkFQwqqB86yJ4WIiEg5GFJgnoIMcEwKERGRkjCkgFOQiYiIlIghBdU9KfklDClERERKwZACINDDfMOeYn0VdBW85ENERKQEDCkA2mkd5GnIWdfKW7k1REREBDCkyIK9zCvmMqQQEREpA0OKRYinCwDg0rWyVm4JERERAQwpMrknpZA9KURERErAkGIR7MmQQkREpCQMKRYhlp6USxyTQkREpAgMKRYcOEtERKQsDCkW1oGz+aWVKK80tnJriIiIiCHFwt3ZAW5aBwBAViFn+BAREbU2hhQLSZIQ6GleeTa7qKKVW0NEREQMKTX4uZlDypVifSu3hIiIiBhSavBz0wIA8hhSiIiIWh1DSg2+1pCiY0ghIiJqbQwpNVhDypUShhQiIqLWxpBSg5+7eUxKno4DZ4mIiFobQ0oNvq6WnhSOSSEiImp1DCk1+Llz4CwREZFSMKTUYJ3dU6KvQlllVSu3hoiIyL4xpNTgqnWAs0YNgJd8iIiIWhtDSg2SJFVPQ2ZIISIialUMKb9jveTDnhQiIqLWxZDyO/LgWU5DJiIialUMKb9jvX/P3oyCVm4JERGRfWNI+Z0/9Q6GWiXh16M5+DEtq7WbQ0REZLcYUn6nZ6gn/nJ7FADgnc2nWrk1RERE9oshpQ6PDe0AtUpCZkEZLl0ra+3mEBER2SWGlDq00zogNsQDALDnHMemEBERtQaGlHoM7NAeALDnXH4rt4SIiMg+MaTUgyGFiIiodTGk1KNvuBccVBIuXSvHmbzi1m4OERGR3WFIqUc7rQNGxPgCAL5OvtDKrSEiIrI/DCkNmDk4EgCwJuUSisoNrdwaIiIi+8KQ0oBBHdsjxt8NZZVGLuxGRET0B2NIaYAkSbi7TzAAYMuJvFZuDRERkX1pdEhJSkrCnXfeiaCgIEiShPXr1ze4f2JiIiRJqvUnJyenqW3+Q42I8QNgnuVToq+C0SRauUVERET2odEhpbS0FD179sSHH37YqOPS09ORnZ0t//Hz82vst24V0X6uCPRwgr7KhO6vbsLIdxJRXmls7WYRERHd8hwae8D48eMxfvz4Rn8jPz8/eHp6Nvq41iZJEkbE+GLlvosAgPP5ZVifloX4Du0R7OUMjZpXzIiIiG6GRoeUpurVqxf0ej26d++OBQsWYPDgwfXuq9frodfr5a91Oh0AwGAwwGBouVk21te63mveHuODlfsuQpIAIYD5a48AAB4aGIZ/TuzcYu1pLTdah1sZa8AaWLEOrAHAGlg1tw7NrZ8khGjyIAtJkrBu3TpMmTKl3n3S09ORmJiIvn37Qq/X47PPPsM333yDvXv3Ii4urs5jFixYgIULF9bavmLFCri4uDS1uU0mBHCyUIK3k8CbadW5zlUj8EZfXvohIiKqS1lZGe6//34UFRXB3d290cff9JBSl+HDhyMsLAzffPNNnc/X1ZMSGhqKq1evNulN1sdgMCAhIQGjR4+GRqO5oWPe3XIGHyaeAwCoJODQKyPhpFG3WJtaQ1PqcKthDVgDK9aBNQBYA6vm1kGn08HHx6fJIeUPu9xTU//+/bFz5856n9dqtdBqtbW2azSam3KyNOZ1543rgrljOqPfG7+hoLQSGQUViA3xbPE2tYabVd+2hDVgDaxYB9YAYA2smlqH5tauVUZ9pqWlITAwsDW+dYtQqyR0DTQnwr//7wju+zQZhWWVrdwqIiKiW0uje1JKSkpw5swZ+euMjAykpaXB29sbYWFhmD9/PrKysvD1118DAN59911ERkaiW7duqKiowGeffYatW7di8+bNLfcuWkHXIHfsPHMVx7PNg3rXpWbJy+gTERFR8zU6pBw4cAC33Xab/PXcuXMBAA8//DCWL1+O7OxsZGZmys9XVlbi+eefR1ZWFlxcXBAbG4vffvvN5jXaoi6BbjZfn71S0kotISIiujU1OqSMGDECDY21Xb58uc3XL7zwAl544YVGN0zpugZ62Hx9NEvXSi0hIiK6NXElsiaK9nPF1D4hGNbJFwBwIluHojID5n6fhskf7ORdk4mIiJqpVWb33ApUKglLpvaEySQQu3AzSvRV6Pla9TibTcdycE/f0FZsIRERUdvGnpRmUqkkdA5wq7V920neNZmIiKg5GFJaQOcag2jf+nMsAGDH6auorDK1VpOIiIjaPF7uaQEzBkXiQn4ZZg2JxLBoX7y1MR1XS/TYf74AunIDsgrLMXNwJNQqqbWbSkRE1GYwpLSAKD9XfDNrgPz1bTG+WJ1yCQt+OobTeeapyZkFZVg4qRsk6caCihACFQYTnB3b9pL7RERETcXLPTfBkyM6op2jWg4oAPB18gX8ejTnhl9jTcoldPnnRvx8+PLNaCIREZHiMaTcBB19XfH21J5QSUAnf1dM6x8GAPjteC6uFOuRV1xx3dfYahl4+/3+ize1rURERErFyz03yfgegdg+7zb4ummx/3wBVu7LxK6zVzHxPztgNAkkvXAb1CoJs77aDyGAV+/shpgas4TOXSkFAOzLKECFwdjm77RMRETUWOxJuYlCvV3gpFGjT7gXHFQScnV65BXrkV9aiX3nC/DtngvYdSYfu8/mY9IHO3EqtxgAYDQJZOSbQ4q+yoS9GQWt+TaIiIhaBUPKH8DF0QE9QmyX0f/teC6WJZ4FAPi4aqGvMuHLXecBAJcLy22mL+84deUPaysREZFSMKT8QQZ2aG/z9Xd7M5FfWonw9i54995eAICf0rJQoq/CuaulNvsmnWZIISIi+8OQ8gcZ1y0AKgkYGu1js33e2BgMjmqPDr7tUFppxLrULGRY7qjcP9IbKgk4lVuCnKLrD7YlIiK6lTCk/EF6hnpiz/yR+PzhfvKibmHeLpjYIxCSJOGBAeEAgP8mncMpy9TluDAvxIZ4AjD3ppRVVrVK24mIiFoDQ8ofyM/dCY4OKrw/rTeGRvvgu0cHyIu7TesfCh9XR2QWlGHF3kwAQAffdhhm6Xl5Yc1hdHt1E5LqGZ+ScuEaPko8A32V8Ybaoq8ywmgSLfCuiIiIbg6GlFYwoUcgvpk1AKHeLvI2F0cHPDGso81+0X6uGNbJV/5aCOCnQ5chhIDRJCCEwNGsIlQYjHju+zS8tTEdr/54TN7/dG4x5qw4iIsFZTavW2EwYsJ7OzBm6XZUGXl/ISIiUiauk6IgDw4Mx8HMayiuqMLgKB/0CvWE0STg4+qIqyWVAIDUzGu44/2dqDAYMbKLPz5NOoex3fyRaQkiq/ZfxKReQRjU0QePf5OCjKulOHypCEkv3CZ/n60n83DWsg5LZkEZQj21f/ybJSIiug6GFAVxdlRj2YN9bLY5qCWsenwgzuSV4MlvD8rhAgDOXjkHANh0LNfmmFnLD+CTh/og42p1EKlp7cGsGq9RypBCRESKxMs9bUCUnxvGdQ9ERHuXBve7Oy4Ewzv5otxgxPy1R+Ckqf7ntS7FX1BaicT0PHn7uSsltV6HiIhICdiT0obEhXvhfL65V+T1yd1QVG7Ascs6+caFwzr5YGQXf/RYsAlZheU2x77xywmMiPGFrrwKVTUGzJ5lSCEiIoViSGlD+oZ7Y+3BLIR5u+DBgeGQJAmbjuXIIaVvhDdctQ4I93aRw4zVj2mX8WPaZbRv5wgAiAvzxMHMQpvLR0RERErCyz1tyF1xwZg5OAJL/hwrT10e3skXsSEeGNXFD8GezgCAzgHu9b5Gfmkl1CoJz47qBAA4k1cCIap7Vg5dLMTtbyei08u/Yu73aTfvzRARUbOdu1KCLSdyr79jG8WelDbESaPGq3d2q7XtpzlDbLZ1DnTDxmPm3pVx3QIQE+CGEC9n/GP9UVRWmTA02gf9IrwhSUBRuQEFZQb52DUpl+Rl+demZuHx4R0aDD1ERNQ6Mq6W4vZ3tgMAfv7LEHQP9rjOEW0Pe1JuQTVDRaRvOzw3uhOm9g3Fk8M6QJKARwZHwtlRLfe8nMwpRkWVef2UdMudmC2L4uKr3ef/6OYTEdF1VFaZ8MQ3B+Svj2QV3fCxQgjoKgzX31EBGFJuQV0Dq0NKeI0F454b3QnHFo6VF4jrGeoJAJiz8hDm71fj3v/uQ3qOOaT8846uAIB1qVnIK67A5mM5SLtYaHNpCABMJoHyyhtb5ZaIiG5chcFY7+frgQsFOJVbPfHB+tl9I15Ycxh9X/8NPx++3Ow23my83HMLCvFyRjtHNUorjQirMW1ZkiS4OFb/k//zjq44f7UUxy7rAEg4nm0+ydUqCff1D8O61CwculSEO/6zE3nFegDAkCgfvD+tN9797RTGdg/A2oNZ+CntMlY9MRDuTg5wcXRAkKWHBgAMRhMS06/g+GUdZg6JgLuT5o8pAhFRG2Y0CUz+YBdKK6uQ8NxwODuqbZ4/eOGazden8248pKxOuQQAmLMiFe5OGpuVzZWGPSm3IJVKwjOjojGqiz/6hHvVu5+/uxNWPxmPNyZ3hbumuockor0LnDRqLLorFhq1JAcUR7UKO89cxSNf7cdXyRcw44v9WJNyCZVGE15aewTj3t2B299JxNaT1YO4Zn65H499fQBLfzuFD7eduXlvmojoFlBhMOLA+QKk5xQjPbcYl66V48CFAlQZTZi/9jDe3pQOwHy/NgCY0isIAHAqtwQmk8Bz36dh6se7bS7nVBlNyNOZ18q6Vlpp8/1eXn/0hu/51hoYUm5Rjw/riM8e7gutg7rB/VwcHXBP3xDcHlR9D5+YADcAQNcgd7xyR1e4OzngtcndMGNwBAAgNbMQAFBZ474/J3OKUWUSqDCY8PjXKTh7pQR5xRXYeeaqvM/29LpvjkhEdKsq0Vfhnz8eRWrmtevvDGDpb6fw54+T8bfVh+RtyWfz8UnSOazcdxEfbDuDK8V6HLR8Dt/bLwwAcKVYj3cS0rEuNQv7z1/DWxtPyscv/L/j6P/mFjz9XQr2ZuQDANq3c4SfmxaZBWX4JvlCC73blsfLPQQAiPEUgOU8jfGvHtMyPT4CDw4Ih0ol4fhlHT5NMi/F76p1gL+7FmqVhEAPZ2w/dQVuTg7o4OuKQxcL8dvxXPi7OwEAgj2dcbmoHCdzipFTVIEAD6c//P0REf2Rvkk+D11FFRzVKnydfAHpOcX4/on46x63y/KL3fFsnbztu72ZqDBU93asPXgJReUGOGlU6BvhhWBPZ2QVluPDbWdtjkk+m4/7+oXhu73mD/cNR3Kw4Yh55mfPUE+M7eaPF/93BG9tTIevmxaTewW3yHtvSQwpBAAIdAb83bTILdbLPSlWKstUny6Bbojxd0N6bjHu7BmEN6Z0hySZ7/9T+r/DeHRoB2QVluPQxULsOpsPPzfzPYHuiA3EnowCHLpYiIGLtiC8vQtGdvbHs6OjsXzXeVwuLMeIGF+M6x543XZWGIzQOqjkdWKIiJSmqNyAf/50DEIAkT7tAADHLutgMgn58/TclRK4OjnAz80JZ/JK8PAX+xDfsX2dA2CLym1n4vx3RwYAoGeIJzRqFUK8nOVVxsd284ensyO+P3ARZ6+U4o0NJ+psY7S/K/7cJxRJp6/il8PZeGZVGoQApvRWVlDh5R4CAEgS8P+mdMWjQyIxsotfPftIePmOLhjZ2Q9zbo+CSiVBkiRE+blizVODMK57AAZHtQcA7M8okO8RNCTaB8NrDMy6kF+GL3ZlYPIHu/DvhFNYtf8invz2INZYBnPVZ8fpK4h7PQEvrTvSQu+aiOjGVFZVj+sAgDxdBd7ccAL3fpKMw5cKbfY9cqkI1omQ1hu9luir5Ju9HrtchLHvJuHPy5JRoq/Cw1/sQ1ZhOdakXILBWD0+UK2y/WXsL7dHAQCullRPZAAgD3ztH+mN9+7rjTf+1B0/zRmMfhHVYxL/enuUzf3fOvm5Qa2S8P59vfH4sA6I8nPFbTF1f/a3JoYUko3o5IuX7+gKjbr+02JotC8+n9FPXmPl9zr5uaF9O0eUG4y4WlIJRwcV+kV4Y2KPQKhVEjr5u+Jfd/eAg0qS//N2tvTcfLD1NIwmUefrns4txkOf70NZpREr912EqZ79iOjWlnTqCpZsOlnvZ0VLulgCLNl8Cscv6/DK+qMYtHgr9mUUQAiB6V/sw6dJ57A3owD/3ZGBkzk6+RezQ78LLVZHLxdBCIHXfz4Og1Egs6AMD32+t9a91lwsM3l6h3pi3tgYuGodsHxmP4ztFiDvI0nA3X1CAACzhkRixWMD8O2sAXDSqOGgViE2xBP/mNhV3n9Sr2Cb3upO/ubPXZVKwksTumD97MHwcFHe7Ete7qEWpVJJGNDBW77uOX1gOJw0asQEuGHfSyPh4ayBg1qFglID/rXxJHqGeuLrR/pj+JJtOJ9fhs92nEMHX1cUllXi7rgQqFQSjlwqwsNf7rP5Phn5pejo63rd9uirjPjXr+noE+6FibHXv5xERMplNJnDAWD+IdvQGAqjSWDj0Rz0jfCSx8c1xvLkC3j7iAOA8zh6uRi7z5oHnD793UF8cH9vnKxxWWbPuXwkn83H1RI9XpvcrVbPitXRLB0cVBL2nCuQt1knInQNdJfHocwcHIFO/m7oGeKJCJ92eGp4R6hUEgw1JiuEebvIyz04adQY1NGn1vfrFeqJ9+7rBYNRIMrPFaO7+uPj7eZxKx392tns66pVZhxQZquoTXtwYDiOZunw4MAwPDa0g7y9vatW/vuTwzugZ4gHugV5wMNZg8eGdsCSTelY9Gv1iHSjSeDefqF46rsUFJRWonuwO8orjTh7pRSpmYU3FFL+m3QOX+zKwBe7MjC++wT5ejARtT37Mqp/uF/v5qhLNqXj4+1n0TfcC2ueGlTr+coqE85bftlRqyRcK61E0ukrGNstAGqVhGXbz8n7WoMEYL7U8u5vpwAAd/UOxs+Hs3HFskwDACz46RicNeaeEI1agsEo4O+uRa5Oj+SzV7H2oPmy9qgu/vjNcs+dfhFeeOWOrpj0wS4AQI9gD5teD+vnlkatwrBOvkg6dQV/H9f5+gUDbIJcXJgnXprQGZ7OjjZrZikZL/dQixvU0QdJL9yGx4d1rHeAqyRJGBTlI3cvPjm8I+aNjYGb1gFaB/Np+U7CKaRcuIZL18rhrFFjxWMDMbKLPwDg4+1nMfeHNPxw4CIqDEakXCjAkk0nUVZZJX+PKqMJK/Zmyl+fzisBESlPhcGIxb+exNqDl2xWtb5aordZ7+PXo9ny3880sHjZztNX8UmSucfgwIVrtab/bjyagyH/2ooxS5Pw5a4MFJRW4s8f78Yzq9Lw0toj2HYyDwWlBjirzW0pN9iuI2LtCZkxOAI9Q6vvl+Pm5ACTAEorjVBJwCt3dEW0n6t82eXQpSLkFesR6dMOH9zfGz1DPaFWSXhhXGf0CPZAl0B3uGkd0DfCu9739u97emL1k/EY36PxPcOSJOHxYR1xT7/QRh/bWtpGlKJbnlolYfZtUXh6REfoq0wY+24SLuSX4a8rUwEAg6Paw91Jg96WpfzP5JXgTF4J1h7Mwp5z+dhx+iquFOuRq9Pj7ak9AQA/pl3G5aLqgW77MvJrzVxqSdlF5XBQqeDrpr3+zkS3sLziCng6O8LR4cZ+D/5ub6Z8GWLnmat4fkwMnv42BYcuFcHLRYNvZg1Al0B3/Ho0Rz7mZHYx9pzLR6CHE8LbV1+6OHKpCE99mwIhzJcwSvRV+HxnBqb1N2L2ioOYPSIK/9l6GsUV5l9oEo7nYlt6ntwzszY1C6kXCwEA8X4CmVWuOFOj18ZBJaHKJDCmqz96BHugX4Q39p83h6D/3Ncbn+08h11n8hHt54bp8RGYHh+ByioTovxccSavBI5qFf51dyycNGp8NbMf8ksr5V7h758YiAqDET6u9X+G+LhqG3z+VsOQQooiSRKcNGq8OK4znv7uoBwyRlhGnfcK85T3ta4NsOVEnjxFb03KJTw3uhMqq0xY8NMxAIDWQQV9lQmv/HgM7205jXfu6WUz26ipfjuei0qjCRN6BKKozIDx7+1AO0cHbJ83Ag4NDD4mUqorxXq8sv4o7oz1b3C/b5LPI9LHFUOia4+DSLlwDfd8kox7+oZi0V09rvs9TSaBr5PPy1+vPZiFlAvXcCHfPBPmWpkBD3y2F8+P6WRzWeXc1VLc9+kehHm7IPFvIyBJwBe7zuPtTekoNxgxINIbL47vjLs+2o1fjmRjz7l8FJYZak3JTc0sRKXRBEky3zX+16M58qD+/n4mOJvc5JDyyOBIvDg+BhIkOYAN7NAeHyWehaeLBkOifdAnwgsfbD2DETHVnzGODipsfGYorpUZoNWo5NuDeLo4wtPFUd7P3UnDW4f8Dj9JSZHGdw9AXI1AYv0PH+jhjImxgegT7oV1Tw+C1kFVaw2BD7aewd9WH0Kxvgr9IrzwxYx+8nNXSyrx9/8dRone9rJQfmkljI2YLHD2Sgke/foAnv7uIPJL9Nh99ioKywzIKiy3WYSpqUr1VVi1LxP5Jfrr70zUQmavOIiNx3Iwe+Whevc5fKkQr/x4DM+sSq11w1EAWJd6CUaTwM+HLqOqxkDP+iSeysOF/DK4OzngyeEdAUAOKN/OGoBeoZ7mdUd+NP/SMWNQhE1PQmZBGVIvFuL/Dmfj9Z+Po9xgxKCO7fH5jH6IC/PCtP6hEML8f7+m50Z1gqeLRl45u1eoJ965pydmDIrAtP5heOfPPRDoAnT2rx77Fu3vCq2D2qaHaGi0D+aP74wPpsVBozYHkJcmdKk1kNVBbe5lZQhpHIYUUiTzmixdoVFLiAvzRIhX9fz+D++Pw/+eGgQ/dyfEhVWvAxBgGcG/JuUiUi5cg6NahfenxdW6f1F2UQX+s+U0ACDlQgFiF27GwMWJeOuQusEP1f9sOY25P6ShssqEZYnVKzueu1pqs/x/zcF9TWEwmnDfp3vw97VH8E7CqWa9FpHVxYIyPP/DIZzMqTtEXy4stzl368gfAGC5ISmQX1qJ7BqXU83HCGw9YZ6GW6yvwo7TV/HvzemY+J8d+OvKVKRZLqPUZB03dk/fUMy5PQrt25l7FkZ29sOQaB8s+XMsao53f3BgGJw0tj+6fj58WR7M+uiQSHw7a4A8W+XVO7vJd3y/s6f5PjcqCbinX4jN58eITn5wcXTAgkndsOiuHpjU0zzmo+Yl4k7+tQfrS5KEJ4Z3rLNXiZqPl3tIseLCvLD1+RFwd67/N4/+kd5IPmeZGnhbR3y1+7x8bXl8jwB5Cf7XJnfDiexiDI5qjzkrUrFqXybmju6El9cfQ5nlVug55RJ2nyvA0E7++Dr5PEK9XeR1CQ5mXsO/LYEhys8V61Kz5DZkXC2Vl7IGgG/3XMDus/l4akRH9GtgAFx9liacwpGsIgDAT2mX8eafrt9lTrcmk0nAJITN5cP8Ej2ulVUiyq9x46vu+3QPsgrLce5qCdY9PbjW8x8l2t4AtKiy1i4AYLMi6olsHYI8nVFeaV4J+lResc04sJnL98t/P3ZZh83Hc7DtbyMQ4O6Ez3ZkwM3JAdss9/S6r38YXLUOePOuHvhsxzm8NLELACDa3w339gvFyn0XMSDSG1F+bhga7YuV+6oHxX+56zwAwNNFg2dGRdvM4nPSqLHqsYHILChDlJ8rwrydEeTpjEAPZ/QJ98LWk+ZQdVvnui8BW9dxkiQ0uubUfI0OKUlJSViyZAlSUlKQnZ2NdevWYcqUKQ0ek5iYiLlz5+LYsWMIDQ3Fyy+/jBkzZjSxyWRPQr1dGnx+QAdvYIv5733CvVBZZcL/+8V8zfnBgeHyftPjIwCYpzX7uh3HlWI95qxIxYlsHdydHDA02ge/HMnB13sysWx7Bg5Y7jD67r29MLlXEJZsTJdf660afwfM99o4b+meBoDz+WU4n1+GnKIKjOrihz0ZBfhiRr961yGoMBiRVVguD577Me2y/FyJvgpXS/S3zEA5k0lAoPZKmr8nhLD7Wx+UVxox7r0kuDtpsO7pQXJQmbl8P45d1mH904PRI8Sj1nEbjmSjuMKAe/qGwmAUePTrA8gv0csLhqVmFiLheC42HMmGEALPje6EKpPAyn0XbV4np1zCvvMF2HHmGqL8XDG2mz/cnDS1Qoq/uxPu/SQZI7v4y70OjmqVfBnFzckBr0zsim/3XsDhS0VYsikdk3oG2YwN6R3miSg/8/k/tluAzaJlAPCPiV0R7Oks94TMGxsDH1dHTO0Tign/2SFfvv3L7dFwq+NyirOjWm7bvLHVU3cHRJp/ifB106J7UO1aAkCghxMW3NkVjg5qeDTwCxPdHI0OKaWlpejZsyceeeQR3HXXXdfdPyMjAxMnTsSTTz6J7777Dlu2bMGjjz6KwMBAjB07tkmNJrKKC/NCqLczHNUqxPi7IdjTGV8ln0dE+3bo+7vLPID5h+PEHoFYvvu8vEbBs6M6oUeQK345koPtp8w9ItYR/M/9kIYPt53BacuofIPJJHeD/6l3MNalZuFX6w27Qjxw6FKR/L2OZ+vk8Smbj+XgrriQOt/Dgp+OYdX+i1jx6AB0C/aQf5gEejghu6gCB85fw7juAXUe+0cwGE14ed1RdA/xwEM1gt/1GE0CKgk2YePxb1KQdvEaNj83HN7tHGsdI4TAP388hg1HsvG/pwYhwqddrX3ailO5xXBQSehgCZ+6CgMc1So4aeq/M/lnO86h0mjC0yOisO98gTw2Y1v6FYzu6o+zV0pw2HKOLd99Hu/c09Pm+PwSPf6yMhVGk8CO01cxuVcwkk7Vvvv4E98cgHXBVn2VCUaTgNEkMLKzH1QqCQnHc7H8lArLThyQj1m0wRGL745Fem51SDmercP2U1dQWmnEL0eycfSyuW0zB0fgE8vNSF+e2AX39AtFTIAbJn+4C2sPZuHc79Y4+XOfuv9vWLlqHTDn9mj5a+92jnh+TAwA4P37e2NfRgGGRPlgcFTjLrn0jfDGW3+ORZSfa4NrKM0YHNmo16WW0+iQMn78eIwfP/6G9//4448RGRmJd955BwDQpUsX7Ny5E0uXLmVIoWZz0qix+dnhkCTzwDRPF0fseOH2Bo+5I9YcUgDzvS8eHhQBY5UBfk4CeRXmexF98lAffLYjAyv3ZZoDioMKCyd1w69Hc5B06gqm9ArCHbGBWJeaJf/GOCLGDyHeLvj1SDZ+v2K39UN54f8dQ9rFQozq4o8ZgyLg6KDCL4fNaz/sOHNV/qAM9nTGiBhffLc3EwfOFyA2xAOr9mXiofiIeqc4H7tchLc2pmPGoAjc1rn+e3AUVxjg4uhw3d4Mq52nr+L7Axfxf4cv44H+YfV+mH+1+zxUEvBQfARK9FUY924SXBzVeH9aHGIC3KCrMGDLyVwIAew9l1/nOg9vb07HN3vMd2z97UQuHq2xGOD12rhyXyYWTOrWqCngBaWVeGHNIYzrHnjdH5SNcbVEj8kf7IKzoxp75o9EQWklxr+XhCg/V6x+chBK9FV4/OsDGNihPf460vzD92SOTu4FHN890OYS4sp9mRjd1R8Jx3Plbf87eAk7Tl/BXXEhmDUkEhuP5UCrVsnLxf98ONvmNe6IDUResR77MgpgEuYxXDm6Cmw8lgMhzAH+7+M7Y41lsbFyo2RpSwBOZOtwPr8Mj31dHVoAyCtLA+ZQeu5KKdQqCY8P6wAXRwfoq4y4p695TY6eoZ6Y0isI69Muy2NTYvzd4N3OsVl3370txq9Z95yxto+U6aaPSUlOTsaoUaNsto0dOxbPPvvszf7WZCecHev/zbQucWFe6B/hjfxSPd69rxfUKgkmScJD0UaY/DrjkaEd4OakwaK7euDhQeHYciIPd8QGIrx9OwyJ8sHqlEuYNTgSV0ttZ94MjfbBUyM64qUJXfDRtjP4rsZCckcvF2HvuXz52rm1y/2pER1RbOmqPpGtk+8c3SXQHf0ivPHd3kxsP3UFu87m40S2DocuFWH5zH7QVVRhX0YBbovxhVolYffZfDzw2V4AwKVrZfWGlPScYkz+cCd6BHvg60cG1Kqd3ggs234Ok3uHyGtP7LUMpiyrNOJCQZl8V9cvd2Ugv6QSz4/phIyrpXjVMuV7Qo9A7D6bj0vXzD1Cd320C5vnDkfGlVK5F+pwVlGtkHKxoMzmVvMnsutfrKumnKIKPPi5+b1H+Lhg3tjO+GzHOfxw4CI+eiCuwXEE/91xDr+dyMOxyzrcHRdc5yUmfZURWofGnWPbTuah3GBEucGIE9k6/Hz4Mq6VGbD//DXkl+ix62w+dp/NR/K5fIzrHoBO/m42Cw+mXLhmEzC2pefh0rUyOaSoVRKMJoG8Yj0+3n4Wm46Zp806Wi4JebpoUFhmwLUy88y35TP7YUSMH77afV4eHDtvbAzWpFySx3Q9HB+BaH83m5Wcgz2d8OH9cTCYTJi/9gjWHjSPxXLWqG0WOLMuBwAAg6N80N5Vi2dGVfd8WM0b1xm/Hs2BvsqEAHcn/PrMUK4CTQ266SElJycH/v62c+79/f2h0+lQXl4OZ+faN6rT6/XQ66t/AOh05i5zg8EAg8FQa/+msr5WS75mW2SPdfhuVl953IP1vApzBUYPDoVGXV2Lju2d0XGI+RKHwWBAgJsGfxlh7voNcNVAJQEmAbRzVKNrQDuoYYJfOwcM7ehtE1KOXCrC25vNY1kGdfTG8cvFSLtYiCe+SZH3OXFZBx/LJZAY/3boH+4BZ43KZqXc7aeu4LMdZ7Fy3yWcu1qKv4/rhP3nr2HLyeou/bNXSnE6pxDFFVUoKK3EgEhv+RLDh1tPo8Jgwv7z1zD7uxR8MK2nfENJg8GAzVkq/LbvDH4+nIMfnx4IB7UKe89V/7A8crEAIR6OOJ1bgoX/dxwAMLG7H7bVuKRwMrsQG2usDFpaacSmI5ehq6ie9n34YmGt8+3HVNu7YJ/ILoLBYMCJ7GIcvazDn+OC6gwRL609LP895XwBKvSVco/EQ5/vQ+Jc8w/C1IuFWPjzCfx9bAwGdvBGeaURK/aae22yiyqQcUWHUC8XZF8zhymDwYCt6Vfw5HepeGpYBzw3ynwHWiEE0i4WIcTLGb5uWlwt0eMvqw5hUMf2mDOiAyRJwpYT1T0ee89dxU81xhmlXsjHwfP5ltcC3tl0Ev+6qzv+d7D6/W8+li1fKuwa6Ibj2cVY9MsJHLSsnPqvu7rjsx0ZMAqB03ml8roe1l69f0/tgZfXH8flogq006rRN8wDBoMBQzp6QaOWEOjhhHFdfdFOIyH5XD783LSYMyISBoMBEV7V97kZ380PRmMVVAD+OSFGDinh7V1wrawSuTo9RnX2xZzbOmLKsj3y+VDfZ4lfOwc8PjQC7287h7t6B8ForILRWOeurc4ePxfr0tw6NLd+kqhrovuNHixJ1x0426lTJ8ycORPz58+Xt23YsAETJ05EWVlZnSFlwYIFWLhwYa3tK1asgItLwwMpif5Irx1UI18voZuXCY93rp6+XGUCvjmtgpcWSMyWIGD+4eogCbzS24hyI/D+MTVKq2x/6HprBQr0EmZ2MqJXe4GzOuDzdDXKqoBoD4FTRbZTLzUqAYNJgloSGOgnkFki4WKp7WuODDJhUrgJ1/Tm9ppg3t8oJIS0EyjUA8MCTbg9SODVlOo23dPBiH4+Ai/uV8MkzNvGBJswMcyE786osO+KZSBnJyMSs1XIKDbv86cIIzZcVEFvNNfl2DUVerc3odIEHLumkuswPFAgyEWgj4+AJAGLD6mRXSZhdLAJCVkqqCWBF3sa8Waa+XepOV2NiPaw/bgqqgT+mVL9u5ZWJfB4ZyPeP1697d4ORgzyF/j4hAonClXwdxb4e08jkvMk/HCuuofk/o5GlBiA/8tUoVd7gYejTXj3qBrnS8zv6+kuRoS7Cqw6p0JqvgpuGoHnuhux74oKGy+Z39dQfxP+FGHCSwfUqLBcLnFUCVSaqv9NxocYcaJQJb8uAAz0M2FPngoqSci1BoBAZ4E7w0349GR1Ozt5mDC7q/lcyy0HFqWpISDBWS1QbpTgqBJY1M+IfVckfH9Ojf6+JjwQVX1u5pYDzmrA3TIk6HCBhABnAT/LR3FZFTB/v7l+L8RWIbjGsKD0IglrzqkwOcIEJ5VAVpmEIQECKgAfHFfjmh54IdYIpwZ+/RUCuFgKBLsAXPPw1ldWVob7778fRUVFcHd3b/TxN70nJSAgALm5uTbbcnNz4e7uXmdAAYD58+dj7ty58tc6nQ6hoaEYM2ZMk95kfQwGAxISEjB69GhoNPY7apt1aHoNfixIxdb0K7hrUFdMGBhm89wky+OId5KQVWieljlzcCTuH9sJANC9TwEe/OIAHB1UcNWqUVBqQIHe/APqgQnDEd7eHMhnVBhQUGaAr6sjFm08hZ1n8uGgknA+vwwGyw+/GYMi8PdxMfhubyYW/HzSph1n9a4YNnIgnl6RBhMKMDDSC7OGRGD2ykO4VGr+4bUlW4O47h1RWnVa7h1KyHFC/7jOMO07Ir9Wlas/+gzpgr/t2wHAHBiqvCNx/nR1r1FasRv0xjL4uWkx/0898OAXB5BV6Wy5g6v5t6oqIWHLZXPbi9sFYtaQCGQnJ0OjlvDm9JHY/e8klOqNWHJEI38fh8AYTBjRAafzSrBy30UculSEXqGeADLRPcgdmQVl0FVU4UCFH4Dq9T4yTD54bnhPzN27HYBAbrmEquBe2H3yDIAK+LtpkVusR1KBq3yJKjVfQufIUJwvqZ5q/r9LLuga5IbUfHPPUrFBwopLHqgwGAGY/3135KowqFdnVBir17exBhQnjQoVBhOKnfyQVV4AQKBPmCdSMguxJ8/80/qvt0fj3S3VU4HvHxyNx4dG4rf3d+Hc1TI4qCS8+9AQRNdYr8O742VkFVagT7gnZn6Vgsm9gjHpjm64UwjcfakInfxc0a6BO9xOqGNbue95HD52AjOm2P5/mADguXpeZ+LEW2tWFj8XzZpbB+uVkKa66SElPj4eGzZssNmWkJCA+Pj4eo/RarXQamsPftNoNDflZLlZr9vWsA6Nr8E/7uiKAR1y8VB8JDT13Kck0sdVDinPjYmBRmP+bzekkz9+mjMYVSaBT7afxaZj5jDv56ZFBz93+Vq9t0YDb8uwisV3V8/mmPtDGtYezIJKAmYM7gCNRoMJscFYvOkUVJKETx7qg5lf7sf5/DJM+igZFwvK4eKoxovju6B3mBe+fkSLHw5cxNqDWdBXmfCvzeYfjrNHdMCWk1dxPFuHF/5nDiih3s64WFCO9NwSJJy8CkON5XnXpGTZLPyVYZmRMrqrP/pG+kCjlpBnWc5co5ZgEpAHd6pVEtYfysZlnfn5ETF+8PVwQecAd6RcuGbzfVIvFuHplYdsBo8eztLJxx3P1mHryTwkW27+9rcxnfD25lNIu1SEn4/kwmgy99gIAfzjx+OorDLBx1WLBZO64anvDsoBpUewO45k6bDygDmgjOrij3NXS3DuSily0/VQqyQsvbcX3vjluLxcuqvWAQM7eOO3E3l4O8G8UOCEHgE2A0v/c19vPP5NCnaeMV/q8XLRYMnUnhi9NAlGk/lOuU+OiLIJKY+PiILWQY3nRsfgLytTMfu2KHQNsZ219ud+1TOu9v9jFFy1DvJ05f4dmnb7h4fiI7Dh2nF+JoCfi1ZNrUNza9fozraSkhKkpaUhLS0NgHmKcVpaGjIzzb9JzZ8/H9OnT5f3f/LJJ3Hu3Dm88MILOHnyJD766CP88MMPeO65+vI4UdvR0dcVTwzv2OCN1J4f0wn9I73xv6fia90ePTbEE3FhXjaDFZdM7XlDgwlnDoqEo4MK9/YLk9eT8XN3ws9/GYJNzw7D0GhfeUXNiwXlcNKosPKxgeht2TawQ3v8+55eeNmyaJbBKNBeKzBzUDj+M60XnDQqmIR5UOT70+IAAFmF5ViTYh47YV1jwjqAcujvVtwc2y0AThq1zXsb2y0A0+PD5f3vjjPP6rAO5pzcy7wORohXdS/rI5bpn9tPXUHC8VyoJHNwcKhRo6HRPjYL5zlpVJg+KAL+7lqbtXP+cns0ugS6o7LK3IP05PAOGBLtI7/WzMER+OGx/hjoZ4LWQQWNWsKc26Pw3r295X0eH9YBk3oG4YsZ/dDOMvB4Qo8APFRjLR5JAp4fEwNrp8JdvYMxNNrXZkZV7zAvdPB1xbT+5tklc26PhpNGjb+P7wxPFw1+eCJeHrB7Z88gHHp1DJ4b3en3p4ENTxdH3jeKbimN7kk5cOAAbrvtNvlr62WZhx9+GMuXL0d2drYcWAAgMjISv/zyC5577jm89957CAkJwWeffcbpx2Q3eod54Ycn6u85BIC7+4Rg49EcPBQffsM3P+wR4oFD/xwD7e8CUs3ZLEOifbDvvDkAzBwcKS8PbvO940KwNOEUKo0mzOhUBTcnDbzdXLB8Zn/sPpuPWYMj4eGiQUffdjh7pVReFv3JER3lmT+A+ZLTjtPmSyFuWgcM7NAegLlH5WROMfzdtVh0Vw+YTED3IA9MjA3Ewcxr+OGAOfS0c1RjZGfzIPvx3QPxY9pl/Kl3MP42thOW786Qp3U/O6oT/joyGnO/T8Pa1Cy0c1Sjd5gXovxcceB8AXzdtPhznxC4O2kwILI9fjpkHrTq6KDCgwPC8NTwjnh7czryS/R4cGA4nDRqLL23F/JL9Hh4UASqqqowraMJnz11O4xCBQ8X82+C/5nWG2kXC/GMZcpwtyAPfD6jH77afR5/uT3asoqpeW2bCd0D0dHXFctn9seWE7mYNzYGzo5qxHdoL99CwToDa8Gd3fDgwHDE+Jv/3Z4c3hFPDOtQ67IJFxIje9TokDJixIg6bypltXz58jqPSU1Nbey3IrIbHX1dsfVvIxp93PWmX4/s4icv5//ksI517uPVzhHrZw+GwVCF0ylJ8vaBHdrLQQMA5o6OwewVBwGYVxEdGuUjryzqrFFjSLQPgjyccLmoArd19pN7lx4f1gH+7k4Y3z1AXg30bsuaJAMj28s/2Md2C5Dfz9hu/tj2txEI93aBSiUh2s9NXkRsimVNjdm3R2HHmauY1DMIjg4qtHfV4vMaN5MEzCsSW0PKw/Hh8LPc3+mVO7ra7GddybQmrYPKdjxGj0BM+N206d/XaP6ELvgm+TzmjTUvNDa8k69N6Pzs4b44nVsCZ0c1OvqaR6Q6qFXoHGA71u5WGddB1Fy8dw/RLaxbkAe+mNEXgR7Oco9AXaL93WAwGHC6gdea0CMA/SO9sS+jADMGRcBBrUKEjwtO5ZagX6Q3tA5q9I/0xnpLD4iVm5PG5hYFNalUEp4ZGY0Ptp3BrKHVq3pKkiSvxwKYe43Sc4vRN9wLYZYBxR19XbH/H6NqvWZNQ6N85XEoT42IanDfljCpZxAm1RF4rJw06jqXsieiujGkEN3ibu/sf/2dboAkSfj84b7YeDRH7nnoFuSBU7klcm/Ba1O6Y9aQDo36QXxf/zDc1z+swX0eHBiOo1lFmDum4TEZvxfW3gUrHxsILxfHOpfhJyJlY0ghohvm5qTB1BrLiP99fGf0j/SWl5R3d9LclJ6CXqGe2PjssCYdW/NyDBG1LQwpRNRk/u5OmHadXhAioqbiXDUiIiJSJIYUIiIiUiSGFCIiIlIkhhQiIiJSJIYUIiIiUiSGFCIiIlIkhhQiIiJSJIYUIiIiUiSGFCIiIlIkhhQiIiJSJIYUIiIiUiSGFCIiIlIkhhQiIiJSpDZxF2QhBABAp9O16OsaDAaUlZVBp9NBo9G06Gu3JawDawCwBlasA2sAsAZWza2D9ee29ed4Y7WJkFJcXAwACA0NbeWWEBERUWMVFxfDw8Oj0cdJoqnx5g9kMplw+fJluLm5QZKkFntdnU6H0NBQXLx4Ee7u7i32um0N68AaAKyBFevAGgCsgVVz6yCEQHFxMYKCgqBSNX6ESZvoSVGpVAgJCblpr+/u7m7XJ6EV68AaAKyBFevAGgCsgVVz6tCUHhQrDpwlIiIiRWJIISIiIkWy65Ci1Wrx6quvQqvVtnZTWhXrwBoArIEV68AaAKyBVWvXoU0MnCUiIiL7Y9c9KURERKRcDClERESkSAwpREREpEgMKURERKRIdh1SPvzwQ0RERMDJyQkDBgzAvn37WrtJTbJo0SL069cPbm5u8PPzw5QpU5Cenm6zz4gRIyBJks2fJ5980mafzMxMTJw4ES4uLvDz88O8efNQVVVls09iYiLi4uKg1WoRFRWF5cuX3+y3d8MWLFhQ6z127txZfr6iogKzZ89G+/bt4erqirvvvhu5ubk2r9HWaxAREVGrBpIkYfbs2QBuzfMgKSkJd955J4KCgiBJEtavX2/zvBAC//znPxEYGAhnZ2eMGjUKp0+fttmnoKAADzzwANzd3eHp6YlZs2ahpKTEZp/Dhw9j6NChcHJyQmhoKN56661abVm9ejU6d+4MJycn9OjRAxs2bGjx91ufhupgMBjw4osvokePHmjXrh2CgoIwffp0XL582eY16jp/Fi9ebLOPkutwvXNhxowZtd7fuHHjbPZp6+fC9WpQ1+eDJElYsmSJvI+izgNhp1atWiUcHR3FF198IY4dOyYee+wx4enpKXJzc1u7aY02duxY8eWXX4qjR4+KtLQ0MWHCBBEWFiZKSkrkfYYPHy4ee+wxkZ2dLf8pKiqSn6+qqhLdu3cXo0aNEqmpqWLDhg3Cx8dHzJ8/X97n3LlzwsXFRcydO1ccP35cvP/++0KtVouNGzf+oe+3Pq+++qro1q2bzXu8cuWK/PyTTz4pQkNDxZYtW8SBAwfEwIEDxaBBg+Tnb4Ua5OXl2bz/hIQEAUBs27ZNCHFrngcbNmwQ//jHP8TatWsFALFu3Tqb5xcvXiw8PDzE+vXrxaFDh8SkSZNEZGSkKC8vl/cZN26c6Nmzp9izZ4/YsWOHiIqKEtOmTZOfLyoqEv7+/uKBBx4QR48eFStXrhTOzs7ik08+kffZtWuXUKvV4q233hLHjx8XL7/8stBoNOLIkSM3vQZCNFyHwsJCMWrUKPH999+LkydPiuTkZNG/f3/Rp08fm9cIDw8Xr732ms35UfNzROl1uN658PDDD4tx48bZvL+CggKbfdr6uXC9GtR879nZ2eKLL74QkiSJs2fPyvso6Tyw25DSv39/MXv2bPlro9EogoKCxKJFi1qxVS0jLy9PABDbt2+Xtw0fPlw888wz9R6zYcMGoVKpRE5Ojrxt2bJlwt3dXej1eiGEEC+88ILo1q2bzXH33nuvGDt2bMu+gSZ69dVXRc+ePet8rrCwUGg0GrF69Wp524kTJwQAkZycLIS4NWrwe88884zo2LGjMJlMQohb/zz4/YeyyWQSAQEBYsmSJfK2wsJCodVqxcqVK4UQQhw/flwAEPv375f3+fXXX4UkSSIrK0sIIcRHH30kvLy85BoIIcSLL74oYmJi5K/vueceMXHiRJv2DBgwQDzxxBMt+h5vRF0/nH5v3759AoC4cOGCvC08PFwsXbq03mPaUh3qCymTJ0+u95hb7Vy4kfNg8uTJ4vbbb7fZpqTzwC4v91RWViIlJQWjRo2St6lUKowaNQrJycmt2LKWUVRUBADw9va22f7dd9/Bx8cH3bt3x/z581FWViY/l5ycjB49esDf31/eNnbsWOh0Ohw7dkzep2bNrPsoqWanT59GUFAQOnTogAceeACZmZkAgJSUFBgMBpv2d+7cGWFhYXL7b5UaWFVWVuLbb7/FI488YnNjTns4D6wyMjKQk5Nj014PDw8MGDDA5t/d09MTffv2lfcZNWoUVCoV9u7dK+8zbNgwODo6yvuMHTsW6enpuHbtmrxPW6kLYP6ckCQJnp6eNtsXL16M9u3bo3fv3liyZInNpb5boQ6JiYnw8/NDTEwMnnrqKeTn58vP2du5kJubi19++QWzZs2q9ZxSzoM2cYPBlnb16lUYjUabD2IA8Pf3x8mTJ1upVS3DZDLh2WefxeDBg9G9e3d5+/3334/w8HAEBQXh8OHDePHFF5Geno61a9cCAHJycuqsh/W5hvbR6XQoLy+Hs7PzzXxr1zVgwAAsX74cMTExyM7OxsKFCzF06FAcPXoUOTk5cHR0rPWB7O/vf933Z32uoX2UUoOa1q9fj8LCQsyYMUPeZg/nQU3WNtfV3prvx8/Pz+Z5BwcHeHt72+wTGRlZ6zWsz3l5edVbF+trKElFRQVefPFFTJs2zeamcX/9618RFxcHb29v7N69G/Pnz0d2djb+/e9/A2j7dRg3bhzuuusuREZG4uzZs3jppZcwfvx4JCcnQ61W29258NVXX8HNzQ133XWXzXYlnQd2GVJuZbNnz8bRo0exc+dOm+2PP/64/PcePXogMDAQI0eOxNmzZ9GxY8c/upk3xfjx4+W/x8bGYsCAAQgPD8cPP/ygqB+cf5TPP/8c48ePR1BQkLzNHs4DapjBYMA999wDIQSWLVtm89zcuXPlv8fGxsLR0RFPPPEEFi1adEssD3/ffffJf+/RowdiY2PRsWNHJCYmYuTIka3YstbxxRdf4IEHHoCTk5PNdiWdB3Z5ucfHxwdqtbrWzI7c3FwEBAS0Uquab86cOfj555+xbds2hISENLjvgAEDAABnzpwBAAQEBNRZD+tzDe3j7u6uyBDg6emJTp064cyZMwgICEBlZSUKCwtt9qn5b34r1eDChQv47bff8Oijjza4361+Hljb3ND/9YCAAOTl5dk8X1VVhYKCghY5N5T0mWINKBcuXEBCQoJNL0pdBgwYgKqqKpw/fx7ArVMHqw4dOsDHx8fm/LeXc2HHjh1IT0+/7mcE0LrngV2GFEdHR/Tp0wdbtmyRt5lMJmzZsgXx8fGt2LKmEUJgzpw5WLduHbZu3VqrG64uaWlpAIDAwEAAQHx8PI4cOWLzH9T6Ida1a1d5n5o1s+6j1JqVlJTg7NmzCAwMRJ8+faDRaGzan56ejszMTLn9t1INvvzyS/j5+WHixIkN7nernweRkZEICAiwaa9Op8PevXtt/t0LCwuRkpIi77N161aYTCY5xMXHxyMpKQkGg0HeJyEhATExMfDy8pL3UXJdrAHl9OnT+O2339C+ffvrHpOWlgaVSiVfArkV6lDTpUuXkJ+fb3P+28O5AJh7Wvv06YOePXted99WPQ8aNcz2FrJq1Sqh1WrF8uXLxfHjx8Xjjz8uPD09bWY1tBVPPfWU8PDwEImJiTZTxsrKyoQQQpw5c0a89tpr4sCBAyIjI0P8+OOPokOHDmLYsGHya1inno4ZM0akpaWJjRs3Cl9f3zqnns6bN0+cOHFCfPjhh4qafvv888+LxMREkZGRIXbt2iVGjRolfHx8RF5enhDCPAU5LCxMbN26VRw4cEDEx8eL+Ph4+fhboQZCmGeqhYWFiRdffNFm+616HhQXF4vU1FSRmpoqAIh///vfIjU1VZ61snjxYuHp6Sl+/PFHcfjwYTF58uQ6pyD37t1b7N27V+zcuVNER0fbTDstLCwU/v7+4qGHHhJHjx4Vq1atEi4uLrWmXDo4OIi3335bnDhxQrz66qt/6BTkhupQWVkpJk2aJEJCQkRaWprN54R1hsbu3bvF0qVLRVpamjh79qz49ttvha+vr5g+fXqbqUNDNSguLhZ/+9vfRHJyssjIyBC//fabiIuLE9HR0aKiokJ+jbZ+Llzv/4MQ5inELi4uYtmyZbWOV9p5YLchRQgh3n//fREWFiYcHR1F//79xZ49e1q7SU0CoM4/X375pRBCiMzMTDFs2DDh7e0ttFqtiIqKEvPmzbNZH0MIIc6fPy/Gjx8vnJ2dhY+Pj3j++eeFwWCw2Wfbtm2iV69ewtHRUXTo0EH+Hkpw7733isDAQOHo6CiCg4PFvffeK86cOSM/X15eLp5++mnh5eUlXFxcxJ/+9CeRnZ1t8xptvQZCCLFp0yYBQKSnp9tsv1XPg23bttV5/j/88MNCCPM05FdeeUX4+/sLrVYrRo4cWas2+fn5Ytq0acLV1VW4u7uLmTNniuLiYpt9Dh06JIYMGSK0Wq0IDg4WixcvrtWWH374QXTq1Ek4OjqKbt26iV9++eWmve/fa6gOGRkZ9X5OWNfQSUlJEQMGDBAeHh7CyclJdOnSRbz55ps2P8CFUHYdGqpBWVmZGDNmjPD19RUajUaEh4eLxx57rNYvpm39XLje/wchhPjkk0+Es7OzKCwsrHW80s4DSQghGtf3QkRERHTz2eWYFCIiIlI+hhQiIiJSJIYUIiIiUiSGFCIiIlIkhhQiIiJSJIYUIiIiUiSGFCIiIlIkhhQiIiJSJIYUIiIiUiSGFCIiIlIkhhQiIiJSJIYUIiIiUqT/D4mthl6b5WyqAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([i[\"step\"] for i in record[\"train\"][::50]], [i[\"loss\"] for i in record[\"train\"][::50]], label=\"train\")\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 推理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T08:03:42.243597600Z",
     "start_time": "2024-07-29T08:03:42.228375900Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-25T02:07:11.288062Z",
     "iopub.status.busy": "2025-01-25T02:07:11.287735Z",
     "iopub.status.idle": "2025-01-25T02:07:11.299703Z",
     "shell.execute_reply": "2025-01-25T02:07:11.299275Z",
     "shell.execute_reply.started": "2025-01-25T02:07:11.288040Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.4502, 0.5498])\n"
     ]
    }
   ],
   "source": [
    "#下面的例子是为了说明temperature\n",
    "my_tensor = torch.tensor([0.4,0.6]) #这里是logits\n",
    "\n",
    "probs1 = F.softmax(my_tensor, dim=-1)\n",
    "print(probs1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T08:17:00.048904100Z",
     "start_time": "2024-07-29T08:17:00.043755800Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-25T02:07:13.346308Z",
     "iopub.status.busy": "2025-01-25T02:07:13.345999Z",
     "iopub.status.idle": "2025-01-25T02:07:13.350726Z",
     "shell.execute_reply": "2025-01-25T02:07:13.350171Z",
     "shell.execute_reply.started": "2025-01-25T02:07:13.346288Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.4750, 0.5250])\n"
     ]
    }
   ],
   "source": [
    "my_tensor = torch.tensor([0.2,0.3])  #现在 temperature是2\n",
    "\n",
    "probs1 = F.softmax(my_tensor, dim=-1)\n",
    "print(probs1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T08:06:39.649844900Z",
     "start_time": "2024-07-29T08:06:39.642147800Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-25T02:07:14.861305Z",
     "iopub.status.busy": "2025-01-25T02:07:14.860988Z",
     "iopub.status.idle": "2025-01-25T02:07:14.873116Z",
     "shell.execute_reply": "2025-01-25T02:07:14.872701Z",
     "shell.execute_reply.started": "2025-01-25T02:07:14.861285Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "概率分布: tensor([0.1000, 0.4500, 0.3500, 0.1000])\n",
      "抽取的样本索引: tensor([2])\n",
      "每个样本对应的概率: tensor([0.3500])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "# 创建一个概率分布，表示每个类别被选中的概率\n",
    "# 这里我们有一个简单的四个类别的概率分布\n",
    "prob_dist = torch.tensor([0.1, 0.45, 0.35, 0.1])\n",
    "\n",
    "# 使用 multinomial 进行抽样\n",
    "# num_samples 表示要抽取的样本数量\n",
    "num_samples = 5\n",
    "\n",
    "# 抽取样本，随机抽样，概率越高，抽到的概率就越高\n",
    "samples = torch.multinomial(prob_dist, 1, replacement=True)\n",
    "\n",
    "print(\"概率分布:\", prob_dist)\n",
    "print(\"抽取的样本索引:\", samples)\n",
    "\n",
    "# 显示每个样本对应的概率\n",
    "print(\"每个样本对应的概率:\", prob_dist[samples])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T08:51:40.360874200Z",
     "start_time": "2024-07-29T08:51:38.070276100Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-25T02:08:13.131374Z",
     "iopub.status.busy": "2025-01-25T02:08:13.131055Z",
     "iopub.status.idle": "2025-01-25T02:08:13.965024Z",
     "shell.execute_reply": "2025-01-25T02:08:13.964587Z",
     "shell.execute_reply.started": "2025-01-25T02:08:13.131354Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_394/612744915.py:26: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  model.load_state_dict(torch.load(\"checkpoints/text_generation/best.ckpt\", map_location=\"cpu\"))\n",
      "  0%|          | 0/1000 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All: I could not satisfy!\n",
      "\n",
      "JULIET:\n",
      "O God, I have seen him for that soul is in the court.\n",
      "\n",
      "Clown:\n",
      "You say you to the"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|█         | 110/1000 [00:00<00:00, 1094.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " contrary.\n",
      "\n",
      "QUEEN MARGARET:\n",
      "That an unstant and weaken'd him.\n",
      "\n",
      "MERCUTIO:\n",
      "The first of the sea,\n",
      "That I should bear my "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 227/1000 [00:00<00:00, 1132.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "boots, I'll give thee first to be a poison.\n",
      "\n",
      "Messenger:\n",
      "Richard yet not your fortune strife, sir, when men made the res"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▍      | 346/1000 [00:00<00:00, 1154.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "t is not so rends:\n",
      "But if the bear better consul, speed us not to visit you.\n",
      "\n",
      "ANGELO:\n",
      "The conceal it the worst and unhap"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 466/1000 [00:00<00:00, 1171.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "py honour's pardon\n",
      "Hath a precious sentence\n",
      "The year shall I see most\n",
      "voice is not by something stoop\n",
      "To loss and hono"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 584/1000 [00:00<00:00, 1148.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ur, for the gates and trumpets shall be the consul,' souls of the consul, that you are come to be hanged up, let him be set\n",
      "Of"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71%|███████   | 710/1000 [00:00<00:00, 1184.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " this tide that I saw the church of the silence,\n",
      "That affection in the sun of little apt from myself.\n",
      "\n",
      "BIANCA:\n",
      "And where they are not so r"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▍ | 848/1000 [00:00<00:00, 1244.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ich in ears and long to the people,\n",
      "Our subjects, and his country.\n",
      "\n",
      "CORIOLANUS:\n",
      "I do not like the name of land,\n",
      "If you had been so dear "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 984/1000 [00:00<00:00, 1280.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "as her children "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:00<00:00, 1216.40it/s]\n"
     ]
    }
   ],
   "source": [
    "def generate_text(model, start_string, max_len=1000, temperature=1.0, stream=True):\n",
    "    input_eval = torch.Tensor([char2idx[char] for char in start_string]).to(dtype=torch.int64, device=device).reshape(1, -1) #bacth_size=1, seq_len长度是多少都可以 （1,5）\n",
    "    hidden = None\n",
    "    text_generated = [] #用来保存生成的文本\n",
    "    model.eval()\n",
    "    pbar = tqdm(range(max_len)) # 进度条\n",
    "    print(start_string, end=\"\")\n",
    "    # no_grad是一个上下文管理器，用于指定在其中的代码块中不需要计算梯度。在这个区域内，不会记录梯度信息，用于在生成文本时不影响模型权重。\n",
    "    with torch.no_grad():\n",
    "        for i in pbar:#控制进度条\n",
    "            logits, hidden = model(input_eval, hidden=hidden)\n",
    "            # 温度采样，较高的温度会增加预测结果的多样性，较低的温度则更加保守。\n",
    "            #取-1的目的是只要最后，拼到原有的输入上\n",
    "            logits = logits[0, -1, :] / temperature\n",
    "            # using multinomial to sampling\n",
    "            probs = F.softmax(logits, dim=-1) #算为概率分布\n",
    "            idx = torch.multinomial(probs, 1).item() #从概率分布中抽取一个样本,取概率较大的那些\n",
    "            input_eval = torch.Tensor([idx]).to(dtype=torch.int64, device=device).reshape(1, -1) #把idx转为tensor\n",
    "            text_generated.append(idx)\n",
    "            if stream:\n",
    "                print(idx2char[idx], end=\"\", flush=True)\n",
    "    return \"\".join([idx2char[i] for i in text_generated])\n",
    "\n",
    "\n",
    "# load checkpoints\n",
    "model.load_state_dict(torch.load(\"checkpoints/text_generation/best.ckpt\", map_location=\"cpu\"))\n",
    "start_string = \"All: \" #这里就是开头，什么都可以\n",
    "res = generate_text(model, start_string, max_len=1000, temperature=0.5, stream=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-23T08:22:23.716047Z",
     "iopub.status.busy": "2025-01-23T08:22:23.715677Z",
     "iopub.status.idle": "2025-01-23T08:22:23.720173Z",
     "shell.execute_reply": "2025-01-23T08:22:23.719579Z",
     "shell.execute_reply.started": "2025-01-23T08:22:23.716020Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"I will assure her sine, and the hand of majesty,\\nWho is as free than his friend, and go with thee to thy father's death,\\nWhich with his son and a hanging head\\nAs thou art so dishonour of mine o'er the sea,\\nWhere you are the honour and the more murdered:\\nThe son of Clarence we must die the people,\\nI mean the business, but we will mouth to show them for the heels.\\n\\nEDWARD:\\nNo, not a word; you must needs mustering years\\nHis face and fear and send the stone, for I know how far off like the world,\\nAnd some of these two breathing state,\\nWho having strength, and will not be absent.\\n\\nRICHARD:\\nBrother, no doubt, sir, be gone.\\n\\nROMEO:\\nWhat says the bastard in the farewell' born in stirr'd with his friends about,\\nAnd be the present at me,\\nAnd now I protest there were fled,\\nAs is the foolish wounds and head is not fourteen;\\nThe exchange of his own royal presence shall be your penitence\\nWith honour, is hither brought\\nThe fairest friend,\\nI am the name of some that hath some privilege at the good fro\""
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
